package spiderRobot;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;

import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import util.SpiderBingRobot;
import util.SpiderSosoRobot;
import util.SpiderSougouRobot;
import util.SpiderYouDaoRobot;
import util.StatisticsWebTimes;

public class UrlSpider {

	/**
	 * 
	 */

	public static void main(String[] args) {
		// 读取xml
		//定义从xml文件中读出的参数
		String dBName[] = new String[4];
		String dBHost[] = new String[4];
		String searchUrlFirst[] = new String[4];
		String searchUrlEnd[] = new String[4];
		String codeNo[] = new String[4];
		int repeatTimes_URL[] = new int[4];
		int interval_URL[] = new int[4];
		String urlPattern[] = new String[4];
		//开始读取xml文件
		SAXBuilder builder = new SAXBuilder(false);
		List sourceSiteParameterList = null;
		try {
			Document doc = builder.build("urlSpider.xml");
			Element parameter = doc.getRootElement();
			sourceSiteParameterList = parameter.getChildren("sourceSite");
		} catch (JDOMException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		// 一次循环拿到一个采集目标网站的配置也就是xml文件中的一个sourceSite
		int i = 0;
		for (Iterator iter = sourceSiteParameterList.iterator(); iter.hasNext();) {		
				Element sourceSite = (Element) iter.next();
				dBName[i] = sourceSite.getChildTextTrim("dBName");
				//System.out.println(dBName[i]);
				dBHost[i] = sourceSite.getAttributeValue("host");
				//System.out.println(dBHost[i]);
				repeatTimes_URL[i] = new Integer(sourceSite.getChildTextTrim("repeatTimesURL"));
				//System.out.println(repeatTimes_URL[i]);
				interval_URL[i] = new Integer(sourceSite.getChildTextTrim("intervalURL"));
				//System.out.println(interval_URL[i]);
				urlPattern[i] = sourceSite.getChildTextTrim("urlPattern");
				//System.out.println(urlPattern[i]);					
				searchUrlFirst[i] = sourceSite.getChildTextTrim("first");
				//System.out.println(searchUrlFirst[i]);
				searchUrlEnd[i] = sourceSite.getChildTextTrim("end");
				//System.out.println(searchUrlEnd[i]);
				codeNo[i] = sourceSite.getChildTextTrim("codeNo");
				//System.out.println(codeNo[i]);
				i++;
		}	
	
			// 抓取链接
		//SpiderRobotWithSpace和SpiderRobot功能类似，只是后者多了除去搜索页内容中空格和换行的功能
		SpiderSougouRobot sogouRobot = new SpiderSougouRobot();
		sogouRobot.setCodeNo(codeNo[0]);
		sogouRobot.setdBName(dBName[0]);
		sogouRobot.setEnd(searchUrlEnd[0]);
		sogouRobot.setFirst(searchUrlFirst[0]);
		sogouRobot.setHost(dBHost[0]);
		sogouRobot.setInterval(interval_URL[0]);
		sogouRobot.setRepeatTimes(repeatTimes_URL[0]);
		sogouRobot.setUrlPattern(urlPattern[0]);
		SpiderYouDaoRobot youDaoRobot = new SpiderYouDaoRobot();
		youDaoRobot.setCodeNo(codeNo[1]);
		youDaoRobot.setdBName(dBName[1]);
		youDaoRobot.setEnd(searchUrlEnd[1]);
		youDaoRobot.setFirst(searchUrlFirst[1]);
		youDaoRobot.setHost(dBHost[1]);
		youDaoRobot.setInterval(interval_URL[1]);
		youDaoRobot.setRepeatTimes(repeatTimes_URL[1]);
		youDaoRobot.setUrlPattern(urlPattern[1]);
		SpiderSosoRobot sosoRobot = new SpiderSosoRobot();
		sosoRobot.setCodeNo(codeNo[2]);
		sosoRobot.setdBName(dBName[2]);
		sosoRobot.setFirst(searchUrlFirst[2]);
		sosoRobot.setHost(dBHost[2]);
		sosoRobot.setInterval(interval_URL[2]);
		sosoRobot.setRepeatTimes(repeatTimes_URL[2]);
		sosoRobot.setUrlPattern(urlPattern[2]);
		SpiderBingRobot bingRobot = new SpiderBingRobot();
		bingRobot.setCodeNo(codeNo[3]);
		bingRobot.setdBName(dBName[3]);
		bingRobot.setEnd(searchUrlEnd[3]);
		bingRobot.setFirst(searchUrlFirst[3]);
		bingRobot.setHost(dBHost[3]);
		bingRobot.setInterval(interval_URL[3]);
		bingRobot.setRepeatTimes(repeatTimes_URL[3]);
		bingRobot.setUrlPattern(urlPattern[3]);
		//Thread sougouSearch = new Thread(sogouRobot);
		//Thread sosoSearch = new Thread(sosoRobot);
		//Thread youdaoSearch = new Thread(youDaoRobot);
		SpiderYouDaoRobot youdaoSearch = new SpiderYouDaoRobot();
		//sougouSearch.start();
		//sosoSearch.start();
		//youdaoSearch.start();
		youdaoSearch.run();
		//抓取完成后，统计每个源网站的出现 
		StatisticsWebTimes swt = new StatisticsWebTimes();
		swt.setdBName(dBName[1]);
		swt.setHost(dBHost[1]);
		swt.statisticsWebTimes();
	}
}
